from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
from tqdm import tqdm
import logging

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# 爬虫函数，使用Selenium抓取豆瓣电影Top250
def crawl_douban_top250_with_selenium():
    # 存储所有电影信息的列表
    movies_list = []
    
    # 配置Chrome选项
    chrome_options = webdriver.ChromeOptions()
    # 添加用户配置，模拟真实浏览器
    chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
    chrome_options.add_argument('--start-maximized')  # 最大化窗口
    chrome_options.add_argument('--disable-blink-features=AutomationControlled')  # 禁用自动化控制标志
    chrome_options.add_argument('--ignore-certificate-errors')
    chrome_options.add_argument('--disable-gpu')
    
    # 初始化浏览器
    logger.info("正在初始化浏览器...")
    driver = webdriver.Chrome(
        service=Service(ChromeDriverManager().install()),
        options=chrome_options
    )
    
    try:
        # 先访问豆瓣主页，建立会话
        logger.info("访问豆瓣主页...")
        driver.get("https://movie.douban.com/")
        time.sleep(random.uniform(3, 5))  # 随机延迟
        
        # 共抓取的页数
        total_pages = 2  # 先抓取前2页试试
        
        # 使用tqdm显示进度条
        for page in tqdm(range(total_pages), desc="抓取进度"):
            # 计算起始索引
            start = page * 25
            url = f"https://movie.douban.com/top250?start={start}&filter="
            
            logger.info(f"正在访问页面: {url}")
            driver.get(url)
            
            # 等待页面加载完成
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CLASS_NAME, "grid_view"))
            )
            
            # 随机滚动页面，模拟人类行为
            for _ in range(3):
                driver.execute_script(f"window.scrollTo(0, {random.randint(100, 800)});")
                time.sleep(random.uniform(0.5, 1.5))
            
            # 获取页面源码
            page_source = driver.page_source
            soup = BeautifulSoup(page_source, 'html.parser')
            
            # 解析电影信息
            logger.info("正在解析电影信息...")
            movie_items = soup.select('ol.grid_view li')
            
            for item in movie_items:
                try:
                    # 电影名称
                    title_tags = item.select('span.title')
                    title_cn = title_tags[0].text.strip() if title_tags else ""
                    title_en = ""
                    if len(title_tags) > 1:
                        title_en = title_tags[1].text.strip().replace('/ ', '')
                    
                    # 导演和主演信息
                    info_tag = item.select_one('div.bd p')
                    if info_tag:
                        info_text = info_tag.text.strip().split('\n')
                        if len(info_text) > 0:
                            director_actor_line = info_text[0].strip()
                            # 提取导演信息
                            if '导演: ' in director_actor_line:
                                director_part = director_actor_line.split('导演: ')[1]
                                if '主演: ' in director_part:
                                    director = director_part.split('主演: ')[0].strip()
                                    actors = director_part.split('主演: ')[1].strip() if len(director_part.split('主演: ')) > 1 else ""
                                else:
                                    director = director_part.strip()
                                    actors = ""
                            else:
                                director = ""
                                actors = ""
                        
                        # 提取年份、国家和类型信息
                        year_country_genre = ""
                        if len(info_text) > 1:
                            year_country_genre = info_text[1].strip()
                        
                        # 分割年份、国家和类型
                        year = ""
                        country = ""
                        genre = ""
                        if year_country_genre:
                            parts = year_country_genre.split('&nbsp;/&nbsp;')
                            if len(parts) > 0:
                                year = parts[0].strip()
                            if len(parts) > 1:
                                country = parts[1].strip()
                            if len(parts) > 2:
                                genre = parts[2].strip()
                    else:
                        director = ""
                        actors = ""
                        year = ""
                        country = ""
                        genre = ""
                    
                    # 评分
                    rating_tag = item.select_one('span.rating_num')
                    rating = rating_tag.text.strip() if rating_tag else ""
                    
                    # 评分人数
                    rating_people_tag = item.find('span', string=lambda text: text and '人评价' in text)
                    rating_people = rating_people_tag.text.strip() if rating_people_tag else ""
                    
                    # 短评
                    quote_tag = item.select_one('span.inq')
                    quote = quote_tag.text.strip() if quote_tag else ""
                    
                    # 将信息添加到列表
                    movies_list.append({
                        '电影名称(中文)': title_cn,
                        '电影名称(外文)': title_en,
                        '导演': director,
                        '主演': actors,
                        '年份': year,
                        '国别': country,
                        '类型': genre,
                        '评分': rating,
                        '评分人数': rating_people,
                        '短评': quote
                    })
                    
                    # 随机休眠，模拟人类阅读
                    time.sleep(random.uniform(0.2, 0.5))
                    
                except Exception as e:
                    logger.error(f"解析电影信息时出错: {e}")
                    continue
            
            # 页面抓取完成后随机休眠较长时间
            sleep_time = random.uniform(8, 15)
            logger.info(f"完成一页处理，休眠 {sleep_time:.2f} 秒")
            time.sleep(sleep_time)
        
    except Exception as e:
        logger.error(f"爬虫执行过程中出错: {e}")
    finally:
        # 关闭浏览器
        logger.info("关闭浏览器...")
        driver.quit()
    
    return movies_list

# 保存数据到Excel
def save_to_excel(movies_list, filename="豆瓣电影Top250.xlsx"):
    if not movies_list:
        logger.warning("没有抓取到数据，无法保存")
        return
    
    # 创建DataFrame
    df = pd.DataFrame(movies_list)
    
    # 保存到Excel文件
    try:
        df.to_excel(filename, index=False, engine='openpyxl')
        logger.info(f"数据已成功保存到 {filename}")
        logger.info(f"共抓取到 {len(movies_list)} 部电影信息")
    except Exception as e:
        logger.error(f"保存Excel文件时出错: {e}")

# 主函数
def main():
    logger.info("开始抓取豆瓣电影Top250信息...")
    start_time = time.time()
    
    # 抓取数据
    movies_list = crawl_douban_top250_with_selenium()
    
    # 保存数据
    save_to_excel(movies_list)
    
    end_time = time.time()
    logger.info(f"抓取完成，总耗时: {end_time - start_time:.2f} 秒")

if __name__ == "__main__":
    main()